%% Author: Administrator
%% Created: 2009-3-9
%% Description: TODO: Add description to image_downloader
-module(image_downloader).

%%
%% Include files
%%
-include("map.hrl").

-include("data_util.hrl").

-define(MAP_HOST, "http://kh.google.com").

%%
%% Exported Functions
%%
-export([start/0, start/2, start/1, stop/0, process/0, download_images/1]).
-export([create_list_file/1]).

%%
%% API Functions
%%
start() ->
    start("list.txt").
    
start(ListFile) ->
    inets:start(),
    {ok, TRef} = timer:apply_interval(60*1000, image_downloader, process, []),
    Pid = spawn(fun() -> download_images(ListFile), timer:cancel(TRef) end),
    register(image_downloader, Pid).
    
start(Level, ListFile) ->
    Pid = spawn(fun() -> download_images(Level, ListFile) end),
    register(image_downloader, Pid).

stop() ->
    image_downloader ! stop.

process() ->
    image_downloader ! process.


download_images(ListFile) ->
%    download_images(18),
    download_images(19, ListFile).

download_images(Level, ListFile) ->
    ListDir = data_importer:get_files_dir(Level-1),
    case file:open(ListDir ++ ListFile, read) of
        {error, _} ->
            exit("the dir " ++ ListDir ++ "error: ");
        {ok, IoDevice} ->
            DownloadDir = data_importer:get_files_dir(Level),
            case file:read_file_info(DownloadDir) of
                {error, enoent} ->
                    ok = file:make_dir(DownloadDir);
                _Any ->
                    ok
            end,
            _HeadLine = io:get_line(IoDevice, ''),
            http:set_options([{proxy, {{"10.0.0.47", 8118},[]}}]),
            try each_download(IoDevice, io:get_line(IoDevice, ''), Level) of
                Count ->
                    io:format("download images: ~p~n", [4 * Count])
            catch 
                throw:_X -> io:format("cannot download\n")
            end,
        
            file:close(IoDevice),
            create_list_file(Level)
    end.


%%
%% Local Functions
%%
create_list_file(Level) ->
    LevelDir = data_importer:get_files_dir(Level),
    {ok, FileNames} = file:list_dir(LevelDir),
    FileName = LevelDir ++ "list.txt",
    {ok, IoDevice} = file:open(FileName, [write]),
    io:format(IoDevice, 
              "ImageFileName    Left_Edge_Longitude    Right_Edge_Longitude   Top_Edge_Latitude      Bottom_Edge_Latitude\r\n", []),
    lists:foreach(fun(ImageFileName) ->
                          [Id, ExtName] = string:tokens(ImageFileName, "."),
                          if
                            ExtName == "jpg" ->
                                #zone{left=Left,right=Right,top=Top,bottom=Bottom} =
                                   geo_util:get_coordinates_from_address(Id),
                                io:format(IoDevice, "~s:   ~p   ~p   ~p   ~p\r\n",
                                    [ImageFileName, Left, Right, Top, Bottom]);
                            true ->
                                ignore
                          end
                  end, FileNames),
    file:close(IoDevice).
    
each_download(IoDevice, Line, Level) ->
    each_download(IoDevice, Line, Level, 0).

each_download(_IoDevice, eof, _Level, Count) ->
    Count;
each_download(IoDevice, Line, Level, Count) ->
    receive
        process ->
            Sum = math:pow(2, 10 + 2 * (Level - 15)),
            io:format("Level ~p, ~.2f%~n", [Level, 100 * Count/Sum]);
        stop ->
            io:format("STOPED BY user\n"),
            exit("stoped")
    after 0 ->
            ok
    end,
    [_FileName | Rest] = string:tokens(Line, ": \n"),
    [Left, Right, Top, Bottom] = 
        lists:map(fun(Str) -> 
                      {Float, []} = string:to_float(Str), 
                      Float 
                  end, Rest),
    Parent = self(),
    spawn(fun() -> 
                  spawn_download(Left, Right, Top, Bottom, Level),
                  Parent ! done
          end),
    receive
        done ->
            ok
    end,                  
    each_download(IoDevice, io:get_line(IoDevice, ''), Level, Count + 4).

spawn_download(Left, Right, Top, Bottom, Level) ->
    process_flag(trap_exit, true),
    CenterLongitude = (Left + Right) / 2,
    CenterLatitude = (Top + Bottom) / 2,
    spawn_link(fun() -> 
                  download_image((Left+CenterLongitude)/2, (Top+CenterLatitude)/2, Level)
          end),
    spawn_link(fun() -> 
                  download_image((Right+CenterLongitude)/2, (Top+CenterLatitude)/2, Level)
          end),
    spawn_link(fun() -> 
                  download_image((Right+CenterLongitude)/2, (Bottom+CenterLatitude)/2, Level)
          end),
    spawn_link(fun() -> 
                  download_image((Left+CenterLongitude)/2, (Bottom+CenterLatitude)/2, Level)
          end),
    receive_loop(4).
        
receive_loop(0) ->
    ok;
receive_loop(N) ->
    receive
        {'EXIT', _Pid, normal} ->
            receive_loop(N-1)
    end.


download_image(Longitude, Latitude, ZoomLevel) ->
    Id = geo_util:get_quadtree_address(Longitude, Latitude, ZoomLevel),
    ImageFileName = data_importer:get_files_dir(ZoomLevel) ++ Id ++ ".jpg",
    case file:read_file_info(ImageFileName) of
        {error, enoent} ->
            Url = ?MAP_HOST ++ "/kh?v=3&t=" ++ Id,
            io:format("url: ~s~n", [Url]),
            RequestHeaders = [
                {"User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6"},
                {"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
                {"Accept-Language", "zh-cn,zh;q=0.5"},
                {"Accept-Encoding", "gzip,deflate"},
                {"Accept-Charset", "gb2312,utf-8;q=0.7,*;q=0.7"},
%                {"Keep-Alive", "300"},
%                {"Connection", "keep-alive"},
                {"Cookie", "PREF=ID=74ab9012e72a09c5:TM=1230542495:LM=1230542495:S=yjBOvaz2r6HlD0bw; SID=DQAAAH0AAACB9TDvGxXnOj3XEOAMaKLOP2cX3ttuiNypL7YbVMhyC5GsBq9s-kQZUEFsww6x5EbYYcQbK9g3MkDy3pnWeGnGMT5Q7bsASKZm_sssWwUN-6s5DwKTdB9o_k7npg1pWSV5lRmfjA0-TZULPkN1rlbfYDDP4bZo1ySp6XFrwr0Apw; NID=17=WaT6qzkn95eWynDSeSNMaD9WbDyztDNafxY9la5L3l-1NvnI6YTIIbEqsgxqLk7pHKm37QP2RwviPHxNHo9hw4oYn-XSPBp33O6JWcBLTA_xPA_Dzvrzz7C5YeTSYgnF"}
            ],
            case http:request(get, {Url,RequestHeaders}, [], []) of
                {ok, {StatusLine, _Headers, Body}} -> 
                    case StatusLine of
                        {_, 502,"Bad Gateway"} ->
                            io:format("Bad Gateway~n"),
                            sleep(),
                            download_image(Longitude, Latitude, ZoomLevel);
                        {_,503,Reason} ->
                            io:format("cannot revoke google! ~s~n", [Reason]),
                            sleep(),
                            download_image(Longitude, Latitude, ZoomLevel);
        %                    throw("has been forbided by google!");
                        {_, 403, _Reason} ->
                            io:format("has been forbided by google!~n"),
                            sleep(),
                            download_image(Longitude, Latitude, ZoomLevel);
        %                    throw(Reason);
                        {_, 404, Reason} ->
                            io:format("404 error: ~s!~n", [Reason]),
                            LevelDir = data_importer:get_files_dir(ZoomLevel),
                            FileName = LevelDir ++ "missed.txt",
                            {ok, IoDevice} = file:open(FileName, [append]),
                            io:format(IoDevice, "~s\r\n", [Url]),
                            file:close(IoDevice),
                            1;
                        {_, 400, Reason} ->
                            io:format("400 error: ~p~n", [Reason]),
                            sleep(),
                            download_image(Longitude, Latitude, ZoomLevel);
%                    throw(Reason);
                        {_, 200, _} ->
                            save_data(ImageFileName, Body),
                            timer:sleep(random:uniform(200)),
                            1
                    end;
                {error, Reason} ->
                    io:format("erroc conn reason: ~p~n", [Reason]),
                    sleep(),
                    download_image(Longitude, Latitude, ZoomLevel)
            end;
        {ok, _Any} ->
            io:format("(~.15f, ~.15f) has downloaded, skip the image: ~s~n", [Longitude, Latitude, Id]),
            0
    end.

sleep() ->
    Sleep = random:uniform(10),
    io:format("~p, ~p: pause for a while: ~p(minutes)~n", [date(), time(), Sleep]),
    receive
        stop ->
            io:format("stoped by user\n"),
            exit("stoped by user")
    after Sleep * 60000 ->
        ok
    end.
%    timer:sleep(Sleep * 60000).

save_data(FileName, Bin) ->
    io:format("save file: ~s~n", [FileName]),
    {ok, IoDevice} = file:open(FileName, [raw, write, binary]),
    ok = file:write(IoDevice, Bin),
    file:close(IoDevice).

